pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
- src/ostbuild/pyostbuild/builtin_build.py \
+ src/ostbuild/pyostbuild/builtin_bin_to_src.py \
+ src/ostbuild/pyostbuild/builtin_branch_prefix.py \
+ src/ostbuild/pyostbuild/builtin_build_components.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
+ src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
- src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
- src/ostbuild/pyostbuild/builtin_query_content.py \
+ src/ostbuild/pyostbuild/builtin_pull_components.py \
+ src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
+ src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
+ src/ostbuild/pyostbuild/jsondb.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/mainloop.py \
{
- "name-prefix": "gnomeos-3.4",
+ "prefix": "gnomeos-3.4",
"architectures": ["i686"],
- "base-prefix": "yocto/gnomeos-3.4",
+ "base": {
+ "name": "yocto",
+ "src": "cgwalters:poky"
+ },
"config-opts": ["--disable-static", "--disable-silent-rules"],
"patches": {"src": "git:/src/ostree",
"branch": "wip/ostbuild-v2",
- "prefix": "gnomeos/3.4"},
+ "subdir": "gnomeos/3.4"},
"components": [
- {"src": "cgwalters:ginitscripts"},
+ {"src": "cgwalters:ginitscripts",
+ "noarch": true},
{"src": "gnome:gtk-doc-stub",
- "component": "devel"},
+ "component": "devel",
+ "noarch": true},
{"src": "git:git://github.com/atgreen/libffi.git"},
gcc. We then import that into an OSTree branch
e.g. "bases/yocto/gnomeos-3.4-i686-devel".
+At present, it's still (mostly) possible to put this data on an ext4
+filesystem and boot into it.
+
We also have a Yocto recipe "ostree-native" which generates (as you
might guess) a native binary of ostree. That binary is used to import
into an "archive mode" OSTree repository. You can see it in
run a build on a source tree, and outputs binaries, which we then add
to the build tree for the next module, and so on.
+The final result of all of this is that the OSTree repository gains
+new commits (which can be downloaded by clients), while still
+retaining old build history.
+
+Yocto details
+-------------
+
+I have a branch of Yocto here:
+
+https://github.com/cgwalters/poky
+
+It has a collection of patches on top of the "Edison" release of
+Yocto, some of which are hacky, others upstreamable. The most
+important part though are the modifications to commit the generated
+root filesystem into OSTree.
+
ostbuild details
----------------
The simple goal of ostbuild is that it only takes as input a
-"manifest" which is basically just a list of components to build. A
-component is a pure metadata file which includes the git repository
+"manifest" which is basically just a list of components to build. You
+can see this here:
+
+http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json
+
+A component is a pure metadata file which includes the git repository
URL and branch name, as well as ./configure flags (--enable-foo).
There is no support for building from "tarballs" - I want the ability
to review all of the code that goes in, and to efficiently store
-source code updates.
+source code updates. It's also just significantly easier from an
+implementation perspective, versus having to maintain a version
+control abstraction layer.
The result of a build of a component is an OSTree branch like
"artifacts/gnomeos-3.4-i686-devel/libxslt/master". Then, a "compose"
process merges together the individual filesystem trees into the final
branches (e.g. gnomeos-3.4-i686-devel).
+Doing local builds
+------------------
+
+This is where you want to modify one (or a few) components on top of
+what comes from the ostree.gnome.org server, and test the result
+locally. I'm working on this.
+
Doing a full build on your system
---------------------------------
+Following this process is equivalent to what we have set up on the
+ostree.gnome.org build server. It will generate a completely new
+repository.
+
srcdir=/src
builddir=/src/build
+++ /dev/null
-#!/bin/bash
-# -*- indent-tabs-mode: nil; -*-
-# Run built image in QEMU
-#
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-set -e
-set -x
-
-SRCDIR=`dirname $0`
-WORKDIR=`pwd`
-
-if test $(id -u) != 0; then
- cat <<EOF
-This script should be run as root.
-EOF
- exit 1
-fi
-
-usage () {
- echo "$0 OSTREE_REPO_PATH"
- exit 1
-}
-
-ARCH=i686
-BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
-
-OBJ=gnomeos-fs.img
-if ! test -f ${OBJ}; then
- cat <<EOF
-Create gnomeos-fs.img like this:
-$ qemu-img create gnomeos-fs.img 6G
-$ mkfs.ext4 -q -F gnomeos-fs.img
-EOF
- exit 1
-fi
-
-if ! test -d ${WORKDIR}/repo/objects; then
- cat <<EOF
-No ./repo/objects found
-EOF
- exit 1
-fi
-
-mkdir -p fs
-umount fs || true
-sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
-mount -o loop gnomeos-fs.img fs
-
-cd fs
-
-TOPROOT_BIND_MOUNTS="home root tmp"
-
-for d in $TOPROOT_BIND_MOUNTS; do
- if ! test -d $d; then
- mkdir -m 0755 $d
- fi
-done
-chmod a=rwxt tmp
-
-if ! test -d ostree; then
- mkdir -p ostree
-
- $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
-fi
-
-rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
-
-current_uname=$(uname -r)
-
-cd ${WORKDIR}
-
-sync
-umount fs
-rmdir fs
-
-ARGS="$@"
-if ! echo $ARGS | grep -q 'ostree='; then
- ARGS="ostree=current $ARGS"
-fi
-ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
-KERNEL=/boot/vmlinuz-${current_uname}
-INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
-
-exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
mkdir -p /ostree
$SRCDIR/gnomeos-setup.sh /ostree
-fi
-cd /ostree
+ cd /ostree
+
+ ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
+ ostree-pull --repo=repo gnome
+ for branch in runtime devel; do
+ ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
+ done
+ ln -sf ${BRANCH_PREFIX}runtime current
-ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
-ostree-pull --repo=repo gnome
-for branch in runtime devel; do
- ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
-done
-ln -sf ${BRANCH_PREFIX}runtime current
+ cd ${WORKDIR}
+fi
uname=$(uname -r)
--- /dev/null
+#!/bin/bash
+# -*- indent-tabs-mode: nil; -*-
+# Create ostree-qemu.img file in the current directory, suitable
+# for booting via qemu.
+#
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+set -x
+
+SRCDIR=`dirname $0`
+WORKDIR=`pwd`
+
+if test $(id -u) != 0; then
+ cat <<EOF
+This script should be run as root.
+EOF
+ exit 1
+fi
+
+usage () {
+ echo "$0"
+ exit 1
+}
+
+OBJ=ostree-qemu.img
+if ! test -f ${OBJ}; then
+ # Hardcoded 6 gigabyte filesystem size here; 6 gigabytes should be
+ # enough for everybody.
+ qemu-img create $OBJ 6G
+ mkfs.ext4 -q -F $OBJ
+fi
+
+mkdir -p fs
+umount fs || true
+sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
+mount -o loop ostree-qemu.img fs
+
+cd fs
+
+if ! test -d ./ostree/repo/objects; then
+ mkdir -p ./ostree
+
+ $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
+fi
+
+mkdir -p ./run ./home ./root ./sys
+mkdir -p ./tmp
+chmod 01777 ./tmp
+
+mkdir -p $(pwd)/ostree/modules
+rsync -a -H -v --delete /ostree/modules/ ./ostree/modules/
+
+cd ..
+umount fs
+
+cat << EOF
+Next, run gnomeos-qemu-pull.sh to copy data.
+EOF
--- /dev/null
+#!/bin/bash
+# -*- indent-tabs-mode: nil; -*-
+# Run built image in QEMU
+#
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+set -x
+
+SRCDIR=`dirname $0`
+WORKDIR=`pwd`
+
+if test $(id -u) != 0; then
+ cat <<EOF
+This script should be run as root.
+EOF
+ exit 1
+fi
+
+usage () {
+ cat <<EOF
+usage: $0 SRC_REPO_PATH CURRENT_REF [REFS...]
+EOF
+ exit 1
+}
+
+SRC_REPO_PATH=$1
+test -n "$SRC_REPO_PATH" || usage
+shift
+
+CURRENT_REF=$1
+test -n "$CURRENT_REF" || usage
+shift
+
+if ! test -f ostree-qemu.img; then
+ cat <<EOF
+ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first
+EOF
+fi
+
+mkdir -p fs
+umount fs || true
+sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
+mount -o loop ostree-qemu.img fs
+
+cd fs
+ostree --repo=${SRC_REPO_PATH} local-clone ./ostree/repo ${CURRENT_REF} "$@"
+
+cd ostree
+ostree --repo=./repo checkout --atomic-retarget ${CURRENT_REF}
+ln -sf ${CURRENT_REF} ${CURRENT_REF}.tmplink
+mv -T ${CURRENT_REF}.tmplink current
+
+cd ${WORKDIR}
+umount fs
--- /dev/null
+#!/bin/bash
+# -*- indent-tabs-mode: nil; -*-
+# Run built image in QEMU
+#
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+set -x
+
+SRCDIR=`dirname $0`
+WORKDIR=`pwd`
+
+if test $(id -u) != 0; then
+ cat <<EOF
+This script should be run as root.
+EOF
+ exit 1
+fi
+
+usage () {
+ echo "$0"
+ exit 1
+}
+
+if ! test -f ostree-qemu.img; then
+ echo "ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first"
+fi
+
+current_uname=$(uname -r)
+
+ARGS="$@"
+if ! echo $ARGS | grep -q 'ostree='; then
+ ARGS="ostree=current $ARGS"
+fi
+ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
+KERNEL=/boot/vmlinuz-${current_uname}
+INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
+
+exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda ostree-qemu.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
mkdir -p ./var/lib/gdm
chown 2:2 ./var/lib/gdm
+mkdir -p ./var/log/gdm
+chown 2:2 ./var/log/gdm
+chmod 01770 ./var/log/gdm
+
+mkdir -p ./var/lib/AccountsService
touch ./var/shadow
chmod 0600 ./var/shadow
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
- if keytype not in ['git']:
+ if keytype not in ['git', 'dirty-git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
--- /dev/null
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildBinToSrc(builtins.Builtin):
+ name = "bin-to-src"
+ short_description = "Turn a binary snapshot into a source snapshot"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def bin_snapshot_to_src(self, bin_snapshot):
+ del bin_snapshot['00ostree-bin-snapshot-version']
+
+ src_snapshot = dict(bin_snapshot)
+ src_snapshot['00ostree-src-snapshot-version'] = 0
+
+ all_architectures = src_snapshot['architecture-buildroots'].keys()
+ # Arbitrarily take first architecture
+ first_arch = all_architectures[0]
+
+ bin_components = src_snapshot['components']
+ src_components = {}
+ src_snapshot['components'] = src_components
+ for archname,rev in bin_components.iteritems():
+ (name, arch) = archname.rsplit('/', 1)
+ if arch != first_arch:
+ continue
+ meta = dict(self.get_component_meta_from_revision(rev))
+ del meta['name']
+ src_components[name] = meta
+
+ for target in src_snapshot['targets']:
+ del target['base']['ostree-revision']
+
+ return src_snapshot
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--bin-snapshot')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+ self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
+
+ snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
+ json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
+
+builtins.register(OstbuildBinToSrc)
--- /dev/null
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildBranchPrefix(builtins.Builtin):
+ name = "prefix-branch"
+ short_description = "Copy current source snapshot to new prefix"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('newprefix')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ if args.newprefix == self.prefix:
+ fatal("Specified prefix %r matches active prefix" % (args.newprefix, ))
+
+ db = self.create_db('src-snapshot', prefix=args.newprefix)
+
+ log("Branching from source snapshot %r" % (self.snapshot_path, ))
+
+ orig_prefix = self.prefix
+
+ forked_snapshot = dict(self.snapshot)
+ forked_snapshot['prefix'] = args.newprefix
+
+ for target in forked_snapshot['targets']:
+ name = target['name']
+ if not name.startswith(orig_prefix):
+ fatal("Mismatched name %r in snapshot" % (name, ))
+ target['name'] = name.replace(orig_prefix, args.newprefix)
+
+ db.store(forked_snapshot)
+
+ run_sync(['ostbuild', 'prefix', args.newprefix],
+ log_initiation=False, log_success=False)
+
+builtins.register(OstbuildBranchPrefix)
+++ /dev/null
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,subprocess,tempfile,re,shutil
-import argparse
-import time
-import urlparse
-import json
-from StringIO import StringIO
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from .subprocess_helpers import run_sync_monitor_log_file
-from . import ostbuildrc
-from . import buildutil
-from . import fileutil
-from . import kvfile
-from . import odict
-from . import vcs
-
-class BuildOptions(object):
- pass
-
-class OstbuildBuild(builtins.Builtin):
- name = "build"
- short_description = "Rebuild all artifacts from the given manifest"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def _get_ostbuild_chroot_args(self, architecture):
- current_machine = os.uname()[4]
- if current_machine != architecture:
- args = ['setarch', architecture]
- else:
- args = []
- args.extend(['ostbuild', 'chroot-compile-one'])
- return args
-
- def _launch_debug_shell(self, architecture, buildroot, cwd=None):
- args = self._get_ostbuild_chroot_args(architecture)
- args.extend(['--buildroot=' + buildroot,
- '--workdir=' + self.workdir,
- '--debug-shell'])
- run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
- fatal("Exiting after debug shell")
-
- def _build_one_component(self, name, component):
- branch = component['branch']
- architecture = component['architecture']
-
- buildname = 'components/%s' % (name, )
-
- current_vcs_version = component['revision']
-
- previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', buildname],
- stderr=open('/dev/null', 'w'),
- none_on_error=True)
- if previous_build_version is not None:
- log("Previous build of '%s' is %s" % (buildname, previous_build_version))
-
- previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', previous_build_version,
- '/_ostbuild-meta.json'],
- log_initiation=True)
- previous_meta = json.loads(previous_metadata_text)
-
- previous_vcs_version = previous_meta['revision']
-
- vcs_version_matches = False
- if previous_vcs_version == current_vcs_version:
- vcs_version_matches = True
- log("VCS version is unchanged from '%s'" % (previous_vcs_version, ))
- if self.buildopts.skip_built:
- return False
- else:
- log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
- else:
- log("No previous build for '%s' found" % (buildname, ))
-
- checkoutdir = os.path.join(self.workdir, 'src')
- component_src = os.path.join(checkoutdir, name)
- run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
-
- artifact_meta = dict(component)
-
- metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
- f = open(metadata_path, 'w')
- json.dump(artifact_meta, f, indent=4, sort_keys=True)
- f.close()
-
- logdir = os.path.join(self.workdir, 'logs', name)
- fileutil.ensure_dir(logdir)
- log_path = os.path.join(logdir, 'compile.log')
- if os.path.isfile(log_path):
- curtime = int(time.time())
- saved_name = os.path.join(logdir, 'compile-prev.log')
- os.rename(log_path, saved_name)
-
- log("Logging to %s" % (log_path, ))
- f = open(log_path, 'w')
- chroot_args = self._get_ostbuild_chroot_args(architecture)
- chroot_args.extend(['--pristine', '--name=' + name])
- if self.buildopts.shell_on_failure:
- ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
- if ecode != 0:
- self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
- else:
- run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
-
- args = ['ostree', '--repo=' + self.repo,
- 'commit', '-b', buildname, '-s', 'Build',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--skip-if-unchanged']
-
- setuid_files = artifact_meta.get('setuid', [])
- statoverride_path = None
- if len(setuid_files) > 0:
- (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
- f = os.fdopen(fd, 'w')
- for path in setuid_files:
- f.write('+2048 ' + path)
- f.close()
- args.append('--statoverride=' + statoverride_path)
-
- component_resultdir = os.path.join(self.workdir, 'results', name)
-
- run_sync(args, cwd=component_resultdir)
- if statoverride_path is not None:
- os.unlink(statoverride_path)
- return True
-
- def _compose(self, target):
- base_name = 'bases/%s' % (target['base']['name'], )
- branch_to_rev = {}
- branch_to_subtrees = {}
-
- contents = [base_name]
- branch_to_subtrees[base_name] = ['/']
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_name])
-
- branch_to_rev[base_name] = base_revision
-
- args = ['ostree', '--repo=' + self.repo, 'rev-parse']
- for component in target['contents']:
- name = component['name']
- contents.append(name)
- args.append('components/%s' % (name, ))
- branch_to_subtrees[name] = component['trees']
- branch_revs_text = run_sync_get_output(args)
- branch_revs = branch_revs_text.split('\n')
-
- for (content, rev) in zip(target['contents'], branch_revs):
- name = content['name']
- branch_to_rev[name] = rev
-
- compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
- if os.path.isdir(compose_rootdir):
- shutil.rmtree(compose_rootdir)
- os.mkdir(compose_rootdir)
-
- resolved_base = dict(target['base'])
- resolved_base['ostree-revision'] = base_revision
- resolved_contents = list(target['contents'])
- for component in resolved_contents:
- component['ostree-revision'] = branch_to_rev[component['name']]
- metadata = {'source': 'ostbuild compose v0',
- 'base': resolved_base,
- 'contents': resolved_contents}
-
- for branch in contents:
- branch_rev = branch_to_rev[branch]
- subtrees = branch_to_subtrees[branch]
- for subtree in subtrees:
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode',
- '--union', '--subpath=' + subtree,
- branch_rev, compose_rootdir])
-
- contents_path = os.path.join(compose_rootdir, 'contents.json')
- f = open(contents_path, 'w')
- json.dump(metadata, f, indent=4, sort_keys=True)
- f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'commit', '-b', target['name'], '-s', 'Compose',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--skip-if-unchanged'], cwd=compose_rootdir)
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--skip-built', action='store_true')
- parser.add_argument('--recompose', action='store_true')
- parser.add_argument('--skip-compose', action='store_true')
- parser.add_argument('--start-at')
- parser.add_argument('--shell-on-failure', action='store_true')
- parser.add_argument('--debug-shell', action='store_true')
- parser.add_argument('components', nargs='*')
-
- args = parser.parse_args(argv)
- self.args = args
-
- self.parse_config()
- self.parse_snapshot()
-
- self.buildopts = BuildOptions()
- self.buildopts.shell_on_failure = args.shell_on_failure
- self.buildopts.skip_built = args.skip_built
-
- build_component_order = []
- if args.recompose:
- pass
- elif len(args.components) == 0:
- tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
- tsorted.reverse()
- build_component_order = tsorted
- else:
- if args.start_at is not None:
- fatal("Can't specify --start-at with component list")
- for name in args.components:
- found = False
- component = self.snapshot['components'].get(name)
- if component is None:
- fatal("Unknown component %r" % (name, ))
- build_component_order.append(name)
-
- start_at_index = -1
- if args.start_at is not None:
- for i,component_name in enumerate(build_component_order):
- if component_name == args.start_at:
- start_at_index = i
- break
- if start_at_index == -1:
- fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
- else:
- start_at_index = 0
-
- for component_name in build_component_order[start_at_index:]:
- component = self.snapshot['components'].get(component_name)
- self._build_one_component(component_name, component)
-
- if not args.skip_compose:
- for target in self.snapshot['targets']:
- self._compose(target)
-
-builtins.register(OstbuildBuild)
--- /dev/null
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class BuildOptions(object):
+ pass
+
+class OstbuildBuildComponents(builtins.Builtin):
+ name = "build-components"
+ short_description = "Build multiple components from given source snapshot"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _get_ostbuild_chroot_args(self, architecture):
+ current_machine = os.uname()[4]
+ if current_machine != architecture:
+ args = ['setarch', architecture]
+ else:
+ args = []
+ args.extend(['ostbuild', 'chroot-compile-one',
+ '--snapshot=' + self.snapshot_path])
+ return args
+
+ def _launch_debug_shell(self, architecture, buildroot, cwd=None):
+ args = self._get_ostbuild_chroot_args(architecture)
+ args.extend(['--buildroot=' + buildroot,
+ '--workdir=' + self.workdir,
+ '--debug-shell'])
+ run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
+ fatal("Exiting after debug shell")
+
+ def _build_one_component(self, basename, component, architecture):
+ branch = component['branch']
+
+ name = '%s/%s' % (basename, architecture)
+ buildname = 'components/%s' % (name, )
+
+ current_vcs_version = component['revision']
+
+ previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', buildname],
+ stderr=open('/dev/null', 'w'),
+ none_on_error=True)
+ if previous_build_version is not None:
+ log("Previous build of '%s' is %s" % (name, previous_build_version))
+
+ previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', previous_build_version,
+ '/_ostbuild-meta.json'],
+ log_initiation=True)
+ previous_meta = json.loads(previous_metadata_text)
+
+ previous_vcs_version = previous_meta['revision']
+
+ vcs_version_matches = False
+ if previous_vcs_version == current_vcs_version:
+ vcs_version_matches = True
+ log("VCS version is unchanged from '%s'" % (previous_vcs_version, ))
+ if self.buildopts.skip_built:
+ return False
+ else:
+ log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
+ else:
+ log("No previous build for '%s' found" % (name, ))
+
+ checkoutdir = os.path.join(self.workdir, 'src')
+ component_src = os.path.join(checkoutdir, basename)
+ run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+ '--clean', '--overwrite', basename], cwd=checkoutdir)
+
+ artifact_meta = dict(component)
+
+ logdir = os.path.join(self.workdir, 'logs', name)
+ fileutil.ensure_dir(logdir)
+ log_path = os.path.join(logdir, 'compile.log')
+ if os.path.isfile(log_path):
+ curtime = int(time.time())
+ saved_name = os.path.join(logdir, 'compile-prev.log')
+ os.rename(log_path, saved_name)
+
+ log("Logging to %s" % (log_path, ))
+ f = open(log_path, 'w')
+ chroot_args = self._get_ostbuild_chroot_args(architecture)
+ chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
+ if self.buildopts.shell_on_failure:
+ ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
+ if ecode != 0:
+ self._launch_debug_shell(architecture, buildroot_name, cwd=component_src)
+ else:
+ run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
+
+ args = ['ostree', '--repo=' + self.repo,
+ 'commit', '-b', buildname, '-s', 'Build',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged']
+
+ setuid_files = artifact_meta.get('setuid', [])
+ statoverride_path = None
+ if len(setuid_files) > 0:
+ (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
+ f = os.fdopen(fd, 'w')
+ for path in setuid_files:
+ f.write('+2048 ' + path)
+ f.close()
+ args.append('--statoverride=' + statoverride_path)
+
+ component_resultdir = os.path.join(self.workdir, 'results', name)
+
+ run_sync(args, cwd=component_resultdir)
+ if statoverride_path is not None:
+ os.unlink(statoverride_path)
+ return True
+
+ def _resolve_refs(self, refs):
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
+
+ def _save_bin_snapshot(self, components, component_architectures):
+ bin_snapshot = dict(self.snapshot)
+
+ del bin_snapshot['00ostree-src-snapshot-version']
+ bin_snapshot['00ostree-bin-snapshot-version'] = 0
+
+ for target in bin_snapshot['targets']:
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+ base['ostree-revision'] = base_revision
+
+ component_refs = []
+ for name in components.iterkeys():
+ for architecture in component_architectures[name]:
+ component_refs.append('components/%s/%s' % (name, architecture))
+
+ new_components = {}
+ resolved_refs = self._resolve_refs(component_refs)
+ for name,rev in zip(components.iterkeys(), resolved_refs):
+ for architecture in component_architectures[name]:
+ archname = '%s/%s' % (name, architecture)
+ new_components[archname] = rev
+
+ bin_snapshot['components'] = new_components
+
+ path = self.get_bin_snapshot_db().store(bin_snapshot)
+ log("Binary snapshot: %s" % (path, ))
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--skip-built', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('--compose', action='store_true')
+ parser.add_argument('--start-at')
+ parser.add_argument('--shell-on-failure', action='store_true')
+ parser.add_argument('--debug-shell', action='store_true')
+ parser.add_argument('components', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
+
+ self.buildopts = BuildOptions()
+ self.buildopts.shell_on_failure = args.shell_on_failure
+ self.buildopts.skip_built = args.skip_built
+
+ required_components = {}
+ component_architectures = {}
+ for target in self.snapshot['targets']:
+ for tree_content in target['contents']:
+ (name, arch) = tree_content['name'].rsplit('/', 1)
+ required_components[name] = self.snapshot['components'][name]
+ if name not in component_architectures:
+ component_architectures[name] = set([arch])
+ else:
+ component_architectures[name].add(arch)
+
+ build_component_order = []
+ if len(args.components) == 0:
+ tsorted = buildutil.tsort_components(required_components, 'build-depends')
+ tsorted.reverse()
+ build_component_order = tsorted
+ else:
+ if args.start_at is not None:
+ fatal("Can't specify --start-at with component list")
+ for name in args.components:
+ found = False
+ component = self.snapshot['components'].get(name)
+ if component is None:
+ fatal("Unknown component %r" % (name, ))
+ build_component_order.append(name)
+
+ start_at_index = -1
+ if args.start_at is not None:
+ for i,component_name in enumerate(build_component_order):
+ if component_name == args.start_at:
+ start_at_index = i
+ break
+ if start_at_index == -1:
+ fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
+ else:
+ start_at_index = 0
+
+ for component_name in build_component_order[start_at_index:]:
+ component = required_components[component_name]
+ architectures = component_architectures[component_name]
+ for architecture in architectures:
+ self._build_one_component(component_name, component, architecture)
+
+ self._save_bin_snapshot(required_components, component_architectures)
+
+ if args.compose:
+ run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
+
+builtins.register(OstbuildBuildComponents)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--overwrite', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot')
+ parser.add_argument('-a', '--active-tree', action='store_true')
parser.add_argument('--clean', action='store_true')
parser.add_argument('components', nargs='*')
self.args = args
self.parse_config()
- self.parse_snapshot()
if len(args.components) > 0:
checkout_components = args.components
else:
checkout_components = [os.path.basename(os.getcwd())]
+ if args.active_tree:
+ self.parse_active_branch()
+ else:
+ self.parse_snapshot(args.prefix, args.snapshot)
+
for component_name in checkout_components:
found = False
- component = self.snapshot['components'].get(component_name)
- if component is None:
- fatal("Unknown component %r" % (component_name, ))
+ component = self.get_component_meta(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
- component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
- component['revision'],
- overwrite=args.overwrite)
+ is_dirty = (keytype == 'dirty-git')
+ if is_dirty:
+ # Kind of a hack, but...
+ if os.path.lexists(checkoutdir):
+ os.unlink(checkoutdir)
+ os.symlink(uri, checkoutdir)
+ else:
+ vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+ component['revision'],
+ overwrite=args.overwrite)
+
if args.clean:
- vcs.clean(keytype, checkoutdir)
+ if is_dirty:
+ log("note: ignoring --clean argument due to \"dirty-git:\" specification")
+ else:
+ vcs.clean(keytype, checkoutdir)
patches = component.get('patches')
if patches is not None:
self.patchdir, patches['branch'],
overwrite=True)
- patch_prefix = patches.get('prefix', None)
- if patch_prefix is not None:
- patchdir = os.path.join(self.patchdir, patch_prefix)
+ patch_subdir = patches.get('subdir', None)
+ if patch_subdir is not None:
+ patchdir = os.path.join(self.patchdir, patch_subdir)
else:
patchdir = self.patchdir
for patch in patches['files']:
patch_path = os.path.join(patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
+
+ metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
+ f = open(metadata_path, 'w')
+ json.dump(component, f, indent=4, sort_keys=True)
+ f.close()
- print "Checked out: %r" % (component_src, )
+ log("Checked out: %r" % (checkoutdir, ))
builtins.register(OstbuildCheckout)
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
+import time
import json
+import hashlib
from . import builtins
from . import buildutil
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
- def _compose_buildroot(self, component_name, dirpath):
+ def _resolve_refs(self, refs):
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
+
+ def _compose_buildroot(self, component_name, architecture):
+ starttime = time.time()
+
+ rootdir_prefix = os.path.join(self.workdir, 'roots')
+ rootdir = os.path.join(rootdir_prefix, component_name)
+ fileutil.ensure_parent_dir(rootdir)
+
+ # Clean up any leftover root dir
+ rootdir_tmp = rootdir + '.tmp'
+ if os.path.isdir(rootdir_tmp):
+ shutil.rmtree(rootdir_tmp)
+
components = self.snapshot['components']
dependencies = buildutil.build_depends(component_name, components)
component = components.get(component_name)
- base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
- component['architecture'],
- 'devel')
+ buildroots = self.snapshot['architecture-buildroots']
+ base_devel_name = 'bases/' + buildroots[architecture]
+
+ refs_to_resolve = [base_devel_name]
checkout_trees = [(base_devel_name, '/')]
for dependency_name in dependencies:
- buildname = 'components/%s' % (dependency_name, )
+ buildname = 'components/%s/%s' % (dependency_name, architecture)
+ refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
+ resolved_refs = self._resolve_refs(refs_to_resolve)
+ ref_to_rev = {}
+ for ref,rev in zip(refs_to_resolve, resolved_refs):
+ ref_to_rev[ref] = rev
+
+ link_cache_dir = os.path.join(self.workdir, 'link-cache')
+ fileutil.ensure_dir(link_cache_dir)
+
+ sha = hashlib.sha256()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in checkout_trees:
+ f.write(ref_to_rev[branch])
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ sha.update(buf)
+ buf = f.read(8192)
+ f.close()
+
+ new_root_cacheid = sha.hexdigest()
+
+ rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
+
+ if os.path.isdir(rootdir):
+ if os.path.isfile(rootdir_cache_path):
+ f = open(rootdir_cache_path)
+ prev_cache_id = f.read().strip()
+ f.close()
+ if prev_cache_id == new_root_cacheid:
+ log("Reusing previous buildroot")
+ os.unlink(tmppath)
+ return rootdir
+ else:
+ log("New buildroot differs from previous")
+
+ shutil.rmtree(rootdir)
+
+ os.mkdir(rootdir_tmp)
+
+ if len(checkout_trees) > 0:
+ log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
+ checkout_trees[-1][0]))
+
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
- for (branch, rootpath) in checkout_trees:
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
- '--union', '--subpath=' + rootpath,
- branch, dirpath])
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--link-cache=' + link_cache_dir,
+ '--user-mode', '--union', '--from-stdin', rootdir_tmp],
+ stdin=open(tmppath))
+
+ os.unlink(tmppath);
+
+ builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
+ os.mkdir(builddir_tmp)
+ os.mkdir(os.path.join(builddir_tmp, 'source'))
+ os.mkdir(os.path.join(builddir_tmp, 'results'))
+ os.rename(rootdir_tmp, rootdir)
+
+ f = open(rootdir_cache_path, 'w')
+ f.write(new_root_cacheid)
+ f.write('\n')
+ f.close()
+
+ endtime = time.time()
+ log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
+
+ return rootdir
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--pristine', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
+ parser.add_argument('--arch', required=True)
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
- self.parse_snapshot()
+ self.parse_snapshot(args.prefix, args.snapshot)
if args.name:
component_name = args.name
else:
- cwd = os.getcwd()
- parent = os.path.dirname(cwd)
- parentparent = os.path.dirname(parent)
- component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+ component_name = self.get_component_from_cwd()
components = self.snapshot['components']
component = components.get(component_name)
shutil.rmtree(child_tmpdir)
fileutil.ensure_dir(child_tmpdir)
- resultdir = os.path.join(self.workdir, 'results', component_name)
+ resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
fileutil.ensure_dir(resultdir)
- rootdir_prefix = os.path.join(workdir, 'roots')
- fileutil.ensure_dir(rootdir_prefix)
- rootdir = os.path.join(rootdir_prefix, component_name)
- fileutil.ensure_parent_dir(rootdir)
- if os.path.isdir(rootdir):
- shutil.rmtree(rootdir)
-
- rootdir_tmp = rootdir + '.tmp'
- builddir = os.path.join(rootdir, 'ostbuild');
- if os.path.isdir(rootdir_tmp):
- shutil.rmtree(rootdir_tmp)
- os.mkdir(rootdir_tmp)
-
- self._compose_buildroot(component_name, rootdir_tmp)
+ rootdir = self._compose_buildroot(component_name, args.arch)
- child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
- run_sync(child_args)
-
- builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
- os.mkdir(builddir_tmp)
- os.mkdir(os.path.join(builddir_tmp, 'source'))
- os.mkdir(os.path.join(builddir_tmp, 'results'))
- os.rename(rootdir_tmp, rootdir)
log("Checked out buildroot: %s" % (rootdir, ))
- sourcedir=os.path.join(builddir, 'source', component_name)
+ sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta.json'])
- child_args.extend(self.metadata.get('config-opts', []))
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
+++ /dev/null
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,re,subprocess,tempfile,shutil
-from StringIO import StringIO
-import argparse
-import json
-
-from . import builtins
-from . import buildutil
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync
-
-class OstbuildChrootRunTriggers(builtins.Builtin):
- name = "chroot-run-triggers"
- short_description = "Run ostree-run-triggers inside a chroot"
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('root')
-
- args = parser.parse_args(argv)
-
- child_args = buildutil.get_base_user_chroot_args()
- child_args.extend(['--mount-proc', '/proc',
- '--mount-bind', '/dev', '/dev',
- args.root,
- '/usr/bin/ostree-run-triggers'])
- env_copy = dict(buildutil.BUILD_ENV)
- env_copy['PWD'] = '/'
- run_sync(child_args, env=env_copy)
-
-builtins.register(OstbuildChrootRunTriggers)
def execute(self, args):
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+ starttime = time.time()
uname=os.uname()
kernel=uname[0].lower()
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
self.makeargs = ['make']
- self.ostbuild_resultdir=None
- self.ostbuild_meta=None
+ self.ostbuild_resultdir='_ostbuild-results'
+ self.ostbuild_meta_path='_ostbuild-meta.json'
chdir = None
+ opt_install = False
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
elif arg.startswith('--ostbuild-meta='):
- self.ostbuild_meta=arg[len('--ostbuild-meta='):]
+ self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
elif arg.startswith('--chdir='):
os.chdir(arg[len('--chdir='):])
- elif arg.startswith('--'):
- self.configargs.append(arg)
else:
self.makeargs.append(arg)
- if self.ostbuild_resultdir is None:
- fatal("Must specify --ostbuild-resultdir=")
- if self.ostbuild_meta is None:
- fatal("Must specify --ostbuild-meta=")
-
- f = open(self.ostbuild_meta)
+ f = open(self.ostbuild_meta_path)
self.metadata = json.load(f)
f.close()
+ self.configargs.extend(self.metadata.get('config-opts', []))
+
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
except OSError, e:
pass
+ endtime = time.time()
+
+ log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+ log("Results placed in %s" % (self.ostbuild_resultdir, ))
+
def _install_and_unlink(self, src, dest):
statsrc = os.lstat(src)
dirname = os.path.dirname(dest)
--- /dev/null
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class OstbuildCompose(builtins.Builtin):
+ name = "compose"
+ short_description = "Build complete trees from components"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _compose_one_target(self, bin_snapshot, target):
+ components = bin_snapshot['components']
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = target['base']['ostree-revision']
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ compose_contents = [(base_revision, '/')]
+ for tree_content in target['contents']:
+ name = tree_content['name']
+ rev = components[name]
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((rev, subpath))
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in compose_contents:
+ f.write(branch)
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ link_cache_dir = os.path.join(self.workdir, 'link-cache')
+ fileutil.ensure_dir(link_cache_dir)
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--link-cache=' + link_cache_dir,
+ '--user-mode', '--no-triggers',
+ '--union', '--from-stdin', compose_rootdir],
+ stdin=open(tmppath))
+ os.unlink(tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(bin_snapshot, f, indent=4, sort_keys=True)
+ f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'commit', '-b', target['name'], '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged'], cwd=compose_rootdir)
+ shutil.rmtree(compose_rootdir)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--bin-snapshot')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
+
+ log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
+
+ for target in self.bin_snapshot['targets']:
+ log("Composing target %r from %u components" % (target['name'],
+ len(target['contents'])))
+ self._compose_one_target(self.bin_snapshot, target)
+
+builtins.register(OstbuildCompose)
--- /dev/null
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+import select,time
+import argparse
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildModifySnapshot(builtins.Builtin):
+ name = "modify-snapshot"
+ short_description = "Change the current source snapshot"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+
+ args = parser.parse_args(argv)
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ component_name = self.get_component_from_cwd()
+ current_meta = self.get_component_meta(component_name)
+
+ new_snapshot = dict(self.snapshot)
+ new_meta = dict(current_meta)
+ if 'patches' in new_meta:
+ del new_meta['patches']
+ new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
+ new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
+
+ new_snapshot['components'][component_name] = new_meta
+
+ db = self.get_src_snapshot_db()
+ path = db.store(new_snapshot)
+ log("Replaced %s with %s %s" % (component_name, new_meta['src'],
+ new_meta['revision']))
+ log("New source snapshot: %s" % (path, ))
+
+builtins.register(OstbuildModifySnapshot)
--- /dev/null
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildPrefix(builtins.Builtin):
+ name = "prefix"
+ short_description = "Display or modify \"prefix\" (build target)"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _set_prefix(self, prefix):
+ f = open(self.path, 'w')
+ f.write(prefix)
+ f.write('\n')
+ f.close()
+ log("Prefix is now %r" % (prefix, ))
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('-a', '--active', action='store_true')
+ parser.add_argument('prefix', nargs='?', default=None)
+
+ args = parser.parse_args(argv)
+
+ self.path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if args.prefix is None and not args.active:
+ if os.path.exists(self.path):
+ f = open(self.path)
+ print "%s" % (f.read().strip(), )
+ f.close()
+ else:
+ log("No currently active prefix")
+ elif args.prefix is not None and args.active:
+ fatal("Can't specify -a with prefix")
+ elif args.prefix is not None:
+ self._set_prefix(args.prefix)
+ else:
+ assert args.active
+
+ self.parse_active_branch()
+
+ active_prefix = self.active_branch_contents['prefix']
+
+ self._set_prefix(active_prefix)
+
+builtins.register(OstbuildPrefix)
--- /dev/null
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import buildutil
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import kvfile
+from . import odict
+
+class OstbuildPullComponents(builtins.Builtin):
+ name = "pull-components"
+ short_description = "Download the component data for active branch"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('targets', nargs='*')
+
+ args = parser.parse_args(argv)
+
+ self.parse_active_branch()
+
+ if len(args.targets) == 0:
+ targets = [self.active_branch]
+ else:
+ targets = args.targets
+
+ tree_contents_list = []
+ for target in targets:
+ tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
+ tree_contents = json.load(open(tree_contents_path))
+ tree_contents_list.append(tree_contents)
+ revisions = set()
+ for tree_contents in tree_contents_list:
+ for content_item in tree_contents['contents']:
+ revisions.add(content_item['ostree-revision'])
+ args = ['ostree-pull', '--repo=' + self.repo]
+ # FIXME FIXME - don't hardcode origin here
+ args.append('gnome')
+ for revision in revisions:
+ args.append(revision)
+ run_sync(args)
+
+builtins.register(OstbuildPullComponents)
+++ /dev/null
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
-# http://people.gnome.org/~walters/docs/build-api.txt
-
-import os,sys,stat,subprocess,tempfile,re,shutil
-import argparse
-from StringIO import StringIO
-import json
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from . import buildutil
-
-class OstbuildQueryContent(builtins.Builtin):
- name = "query-content"
- short_description = "Output metadata from a component"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--branch', required=True)
- parser.add_argument('--component')
-
- args = parser.parse_args(argv)
- self.args = args
- self.parse_config()
-
- contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', args.branch, 'contents.json'])
-
- if args.component is None:
- sys.stdout.write(contents_json_text)
- else:
- contents = json.loads(contents_json_text)
- contents_list = contents['contents']
- found = False
- for content in contents_list:
- if content['name'] != args.component:
- found = True
- break
- if not found:
- fatal("Unknown component '%s'" % (args.component, ))
- ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', content['ostree-revision'],
- '/_ostbuild-meta.json'])
- sys.stdout.write(ostbuildmeta_json)
-
-builtins.register(OstbuildQueryContent)
import copy
import argparse
import json
+import time
import urlparse
from StringIO import StringIO
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
+from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
builtins.Builtin.__init__(self)
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+ # FIXME - remove "name" from parameter list here - hash uri?
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
self.args = args
self.parse_config()
+ self.repo = ostbuildrc.get_key('repo')
manifest_path = self.ostbuildrc.get_key('manifest')
self.manifest = json.load(open(manifest_path))
component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
- name_prefix = snapshot['name-prefix']
- base_prefix = snapshot['base-prefix']
-
manifest_architectures = snapshot['architectures']
+ ostree_prefix = snapshot['prefix']
+ base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
+
+ snapshot['architecture-buildroots'] = {}
+ for architecture in manifest_architectures:
+ snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
+
components_by_name = {}
component_ordering = []
- build_prev_component_by_arch = {}
- runtime_prev_component_by_arch = {}
- runtime_components_by_arch = {}
- devel_components_by_arch = {}
- for architecture in manifest_architectures:
- runtime_components_by_arch[architecture] = []
- devel_components_by_arch[architecture] = []
+ build_prev_component = None
+ runtime_prev_component = None
+ runtime_components = []
+ devel_components = []
- for component in component_source_list:
- component_architectures = component.get('architectures', manifest_architectures)
- for architecture in component_architectures:
- component_binary = copy.deepcopy(component)
- source_name = component['name']
- binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
- component_binary['name'] = binary_name
- component_binary['architecture'] = architecture
+ builds = {}
- components_by_name[binary_name] = component_binary
+ for component in component_source_list:
+ base_name = component['name']
+ name = '%s/%s' % (ostree_prefix, base_name)
+ component['name'] = name
- prev_component = build_prev_component_by_arch.get(architecture)
- if prev_component is not None:
- component_binary['build-depends'] = [prev_component['name']]
- build_prev_component_by_arch[architecture] = component_binary
+ components_by_name[name] = component
- is_runtime = component.get('component', 'runtime') == 'runtime'
+ if build_prev_component is not None:
+ component['build-depends'] = [build_prev_component['name']]
+ build_prev_component = component
- prev_component = runtime_prev_component_by_arch.get(architecture)
- if prev_component is not None:
- component_binary['runtime-depends'] = [prev_component['name']]
+ is_runtime = component.get('component', 'runtime') == 'runtime'
- if is_runtime:
- runtime_prev_component_by_arch[architecture] = component_binary
+ if runtime_prev_component is not None:
+ component['runtime-depends'] = [runtime_prev_component['name']]
- if is_runtime:
- runtime_components_by_arch[architecture].append(component_binary)
- devel_components_by_arch[architecture].append(component_binary)
+ if is_runtime:
+ runtime_prev_component = component
+ runtime_components.append(component)
+ devel_components.append(component)
- if 'architectures' in component_binary:
- del component_binary['architectures']
+ is_noarch = component.get('noarch', False)
+ if is_noarch:
+ # Just use the first specified architecture
+ component_arches = [manifest_architectures[0]]
+ else:
+ component_arches = component.get('architectures', manifest_architectures)
+ builds[name] = component_arches
# We expanded these keys
del snapshot['config-opts']
targets_list = []
snapshot['targets'] = targets_list
- for architecture in manifest_architectures:
- for target_component_type in ['runtime', 'devel']:
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in manifest_architectures:
target = {}
targets_list.append(target)
- target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
+ target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', 'bases/%s' % (base_ref, )])
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
- target_components = runtime_components_by_arch[architecture]
+ target_components = runtime_components
else:
- target_components = devel_components_by_arch[architecture]
+ target_components = devel_components
contents = []
for component in target_components:
- name = component['name']
- component_ref = {'name': name}
+ builds_for_component = builds[component['name']]
+ if architecture not in builds_for_component:
+ continue
+ binary_name = '%s/%s' % (component['name'], architecture)
+ component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
del component['name']
snapshot['components'] = components_by_name
- out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
- f = open(out_snapshot, 'w')
- json.dump(snapshot, f, indent=4, sort_keys=True)
- f.close()
- print "Created: %s" % (out_snapshot, )
+ snapshot['00ostree-src-snapshot-version'] = 0
+
+ current_time = time.time()
+
+ src_db = self.get_src_snapshot_db()
+ path = src_db.store(snapshot)
+ log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildResolve)
import os
import sys
+import stat
import argparse
import json
from . import ostbuildrc
+from . import fileutil
+from . import jsondb
from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
_all_builtins = {}
name = None
short_description = None
+ def __init__(self):
+ self._meta_cache = {}
+ self.prefix = None
+ self.manifest = None
+ self.snapshot = None
+ self.bin_snapshot = None
+ self.repo = None
+ self.ostree_dir = self._find_ostree_dir()
+ (self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+ self._src_snapshots = None
+ self._bin_snapshots = None
+
+ def _find_ostree_dir(self):
+ for path in ['/ostree', '/sysroot/ostree']:
+ if os.path.isdir(path):
+ return path
+ return None
+
+ def _find_active_branch(self):
+ if self.ostree_dir is None:
+ return (None, None)
+ current_path = os.path.join(self.ostree_dir, 'current')
+ while True:
+ try:
+ target = os.path.join(self.ostree_dir, current_path)
+ stbuf = os.lstat(target)
+ except OSError, e:
+ current_path = None
+ break
+ if not stat.S_ISLNK(stbuf.st_mode):
+ break
+ current_path = os.readlink(target)
+ if current_path is not None:
+ basename = os.path.basename(current_path)
+ return basename.rsplit('-', 1)
+ else:
+ return (None, None)
+
+ def get_component_from_cwd(self):
+ cwd = os.getcwd()
+ parent = os.path.dirname(cwd)
+ parentparent = os.path.dirname(parent)
+ return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+
def parse_config(self):
self.ostbuildrc = ostbuildrc
- self.repo = ostbuildrc.get_key('repo')
+
self.mirrordir = ostbuildrc.get_key('mirrordir')
if not os.path.isdir(self.mirrordir):
fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = ostbuildrc.get_key('workdir')
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
+
+ self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
self.patchdir = os.path.join(self.workdir, 'patches')
- def parse_manifest(self):
- self.manifest_path = ostbuildrc.get_key('manifest')
- self.manifest = json.load(open(self.manifest_path))
- self.name_prefix = self.manifest['name-prefix']
+ def parse_active_branch(self):
+ if self.ostree_dir is None:
+ fatal("/ostree directory not found")
+ repo_path = os.path.join(self.ostree_dir, 'repo')
+ if not os.path.isdir(repo_path):
+ fatal("Repository '%s' doesn't exist" % (repo_path, ))
+ self.repo = repo_path
+ if self.active_branch is None:
+ fatal("No \"current\" link found")
+ branch_path = os.path.join(self.ostree_dir, self.active_branch)
+ contents_path = os.path.join(branch_path, 'contents.json')
+ f = open(contents_path)
+ self.active_branch_contents = json.load(f)
+ f.close()
+
+ def get_component_snapshot(self, name):
+ found = False
+ for content in self.active_branch_contents['contents']:
+ if content['name'] == name:
+ found = True
+ break
+ if not found:
+ fatal("Unknown component '%s'" % (name, ))
+ return content
+
+ def get_component_meta_from_revision(self, revision):
+ text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', revision,
+ '/_ostbuild-meta.json'])
+ return json.loads(text)
+
+ def get_component_meta(self, name):
+ assert self.repo is not None
- def parse_snapshot(self):
- self.parse_manifest()
- snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
- self.snapshot = json.load(open(snapshot_path))
+ if self.snapshot is not None:
+ return self.snapshot['components'][name]
+
+ meta = self._meta_cache.get(name)
+ if meta is None:
+ content = self.get_component_snapshot(name)
+ meta = self.get_component_meta_from_revision(content['ostree-revision'])
+ self._meta_cache[name] = meta
+ return meta
+
+ def get_prefix(self):
+ if self.prefix is None:
+ path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if not os.path.exists(path):
+ fatal("No prefix set; use \"ostbuild prefix\" to set one")
+ f = open(path)
+ self.prefix = f.read().strip()
+ f.close()
+ return self.prefix
+
+ def create_db(self, dbsuffix, prefix=None):
+ if prefix is None:
+ target_prefix = self.get_prefix()
+ else:
+ target_prefix = prefix
+ name = '%s-%s' % (target_prefix, dbsuffix)
+ fileutil.ensure_dir(self.snapshot_dir)
+ return jsondb.JsonDB(self.snapshot_dir, prefix=name)
+
+ def get_src_snapshot_db(self):
+ if self._src_snapshots is None:
+ self._src_snapshots = self.create_db('src-snapshot')
+ return self._src_snapshots
+
+ def get_bin_snapshot_db(self):
+ if self._bin_snapshots is None:
+ self._bin_snapshots = self.create_db('bin-snapshot')
+ return self._bin_snapshots
+
+ def parse_snapshot(self, prefix, path):
+ if prefix is not None:
+ self.prefix = prefix
+ self.repo = ostbuildrc.get_key('repo')
+ if path is None:
+ latest_path = self.get_src_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
+ self.snapshot_path = latest_path
+ else:
+ self.snapshot_path = path
+ self.snapshot = json.load(open(self.snapshot_path))
+ src_ver = self.snapshot['00ostree-src-snapshot-version']
+ if src_ver != 0:
+ fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver)
+
+ def parse_bin_snapshot(self, prefix, path):
+ if prefix is not None:
+ self.prefix = prefix
+ self.repo = ostbuildrc.get_key('repo')
+ if path is None:
+ latest_path = self.get_bin_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
+ self.bin_snapshot_path = latest_path
+ else:
+ self.bin_snapshot_path = path
+ self.bin_snapshot = json.load(open(self.bin_snapshot_path))
+ bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
+ if bin_ver != 0:
+ fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver)
def execute(self, args):
raise NotImplementedError()
return None
def get_all():
- return _all_builtins.itervalues()
+ return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))
--- /dev/null
+#
+# Copyright (C) 2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+import time
+import tempfile
+import re
+import shutil
+import hashlib
+import json
+
+class JsonDB(object):
+ def __init__(self, dirpath, prefix):
+ self._dirpath = dirpath
+ self._prefix = prefix
+ self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
+
+ def _cmp_match_by_version(self, a, b):
+ # Note this is a reversed comparison; bigger is earlier
+ a_major = a[0]
+ a_minor = a[1]
+ b_major = b[0]
+ b_minor = b[1]
+
+ c = cmp(b_major, a_major)
+ if c == 0:
+ return cmp(b_minor, a_minor)
+ return 0
+
+ def _get_all(self):
+ result = []
+ for fname in os.listdir(self._dirpath):
+ if not (fname.startswith(self._prefix) and fname.endswith('.json')):
+ continue
+
+ path = os.path.join(self._dirpath, fname)
+ match = self._version_csum_re.search(fname)
+ if not match:
+ raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
+ path)
+ result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
+ result.sort(self._cmp_match_by_version)
+ return result
+
+ def get_latest(self):
+ path = self.get_latest_path()
+ if path is None:
+ return None
+ return json.load(open(path))
+
+ def get_latest_path(self):
+ files = self._get_all()
+ if len(files) == 0:
+ return None
+ return os.path.join(self._dirpath, files[0][3])
+
+ def store(self, obj):
+ files = self._get_all()
+ if len(files) == 0:
+ latest = None
+ else:
+ latest = files[0]
+
+ current_time = time.gmtime()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
+ os.close(fd)
+ f = open(tmppath, 'w')
+ json.dump(obj, f, indent=4, sort_keys=True)
+ f.close()
+
+ csum = hashlib.sha256()
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ csum.update(buf)
+ buf = f.read(8192)
+ f.close()
+ digest = csum.hexdigest()
+
+ if latest is not None:
+ if digest == latest[2]:
+ return latest[3]
+ latest_version = (latest[0], latest[1])
+ else:
+ latest_version = (current_time.tm_year, 0)
+ target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
+ latest_version[1] + 1, digest)
+ target_path = os.path.join(self._dirpath, target_name)
+ os.rename(tmppath, target_path)
+ return target_path
+
+
+
import argparse
from . import builtins
-from . import builtin_build
+from . import builtin_bin_to_src
+from . import builtin_build_components
+from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
-from . import builtin_chroot_run_triggers
+from . import builtin_compose
from . import builtin_compile_one
-from . import builtin_query_content
+from . import builtin_pull_components
+from . import builtin_prefix
from . import builtin_resolve
+from . import builtin_modify_snapshot
from . import builtin_status
def usage(ecode):
import os
import sys
-def log(msg):
- fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
+def log(msg, prefix=None):
+ if prefix is None:
+ prefix_target = ''
+ else:
+ prefix_target = prefix
+ fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
sys.stdout.write(fullmsg)
sys.stdout.flush()
def fatal(msg):
- log(msg)
+ log(msg, prefix="FATAL: ")
sys.exit(1)
return None
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
- log_success=True, log_initiation=True, stdout=None,
+ log_success=True, log_initiation=True, stdin=None, stdout=None,
stderr=None):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
- if keep_stdin:
+ if stdin is not None:
+ stdin_target = stdin
+ elif keep_stdin:
stdin_target = sys.stdin
else:
stdin_target = open('/dev/null', 'r')
return dest
def clean(keytype, checkoutdir):
- assert keytype == 'git'
+ assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
projects / ostree.git / commitdiff